import requests
import os
import lxml.html


etree = lxml.html.etree
class Spider(object):
    def book(self):
        url = "https://www.qidian.com/all"
        r = requests.get(url)
        html = etree.HTML(r.text)  # 获取html
        # 找到当前页小说书名 返回一个列表,子元素是str----------------------
        title = html.xpath('//div[@class="book-mid-info"]/h4/a/text()')
        # 找到小说的超链接 返回一个列表,子元素是str-------------------
        src = html.xpath('//div[@class="book-mid-info"]/h4/a/@href')
        for title_t, src_s in zip(title, src):
            if not os.path.exists(title_t):
                os.mkdir(title_t)
            print(title_t, src_s)
            self.file_name(title_t, src_s)


    def file_name(self, title_t, src_s):
        print("爬虫正在抓取")
        url = 'https:' + src_s
        r = requests.get(url)
        html = etree.HTML(r.text)
        title = html.xpath('//ul[@class="cf"]/li/a/text()')
        src = html.xpath('//ul[@class="cf"]/li/a/@href')
        for title_t2, src_s in zip(title, src):
            print(title_t2, src_s)
            self.finally_(title_t, src_s)

    def finally_(self, t, s):
        r = requests.get('http:' + s)
        html = etree.HTML(r.text)
        list_data = html.xpath('//div[@class="read-content j_readContent"]/p/text()')
        for data in list_data:
            with open(t + "/" + t, "a", encoding='utf-8') as f:
                f.write(data)


spider = Spider()
spider.book()
